import os
import cv2
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
import pandas as pd
import torch
import torch.nn as nn
import torchvision.transforms as transforms
import torch.optim as optim
import torchvision.models as models
import time
import itertools


from sklearn.preprocessing import LabelEncoder
from sklearn.preprocessing import StandardScaler
from sklearn.metrics import confusion_matrix
from torch.utils.data import DataLoader, Dataset
from torchvision import models
from PIL import Image
from torchvision.models import ResNet50_Weights
from sklearn.model_selection import train_test_split
from torchvision import transforms
from pytorch_grad_cam import GradCAM
from pytorch_grad_cam.utils.image import preprocess_image
from sklearn.utils.class_weight import compute_class_weight
from torch.optim.lr_scheduler import StepLR
import torch.optim.lr_scheduler as lr_scheduler
from torch.optim import Adam

# Define BASE_DIR and derived paths
BASE_DIR = r"E:\afeka\finalProject"
MEDIA_ROOT = os.path.join(BASE_DIR, 'JupyterLab')
ISIC_DIR = os.path.join(MEDIA_ROOT, 'ISIC-images')

metadata_path = os.path.join(ISIC_DIR, 'metadata.csv')
root_dir = ISIC_DIR  

# Load metadata
metadata = pd.read_csv(metadata_path)

# Fix image filenames (append .jpg)
metadata["image_name"] = metadata["isic_id"] + ".jpg"

# Check if images exist
missing_files = [f for f in metadata["image_name"] if not os.path.exists(os.path.join(root_dir, f))]

if missing_files:
    print("Missing files:", missing_files[:5]) 
else:
    print("All images exist.")

All images exist.

print(metadata.head())

        isic_id                                        attribution  \
0  ISIC_0024306  ViDIR Group, Department of Dermatology, Medica...   
1  ISIC_0024307  ViDIR Group, Department of Dermatology, Medica...   
2  ISIC_0024308  ViDIR Group, Department of Dermatology, Medica...   
3  ISIC_0024309  ViDIR Group, Department of Dermatology, Medica...   
4  ISIC_0024310  ViDIR Group, Department of Dermatology, Medica...   

  copyright_license  age_approx anatom_site_general anatom_site_special  \
0          CC-BY-NC        45.0                 NaN                 NaN   
1          CC-BY-NC        50.0     lower extremity                 NaN   
2          CC-BY-NC        55.0                 NaN                 NaN   
3          CC-BY-NC        40.0                 NaN                 NaN   
4          CC-BY-NC        60.0      anterior torso                 NaN   

  benign_malignant  concomitant_biopsy diagnosis diagnosis_1  \
0           benign               False     nevus      Benign   
1           benign               False     nevus      Benign   
2           benign               False     nevus      Benign   
3           benign               False     nevus      Benign   
4        malignant                True  melanoma   Malignant   

                                       diagnosis_2    diagnosis_3  \
0                Benign melanocytic proliferations          Nevus   
1                Benign melanocytic proliferations          Nevus   
2                Benign melanocytic proliferations          Nevus   
3                Benign melanocytic proliferations          Nevus   
4  Malignant melanocytic proliferations (Melanoma)  Melanoma, NOS   

             diagnosis_confirm_type   image_type   lesion_id  melanocytic  \
0  serial imaging showing no change  dermoscopic  IL_7252831         True   
1  serial imaging showing no change  dermoscopic  IL_6125741         True   
2  serial imaging showing no change  dermoscopic  IL_3692653         True   
3  serial imaging showing no change  dermoscopic  IL_0959663         True   
4                    histopathology  dermoscopic  IL_8194852         True   

      sex        image_name  
0    male  ISIC_0024306.jpg  
1    male  ISIC_0024307.jpg  
2  female  ISIC_0024308.jpg  
3    male  ISIC_0024309.jpg  
4    male  ISIC_0024310.jpg

print(metadata.columns.tolist())  # List all column names
print(metadata["diagnosis"].value_counts())  # Check label distribution

['isic_id', 'attribution', 'copyright_license', 'age_approx', 'anatom_site_general', 'anatom_site_special', 'benign_malignant', 'concomitant_biopsy', 'diagnosis', 'diagnosis_1', 'diagnosis_2', 'diagnosis_3', 'diagnosis_confirm_type', 'image_type', 'lesion_id', 'melanocytic', 'sex', 'image_name']
diagnosis
nevus                         7737
pigmented benign keratosis    1338
melanoma                      1305
basal cell carcinoma           622
squamous cell carcinoma        229
vascular lesion                180
dermatofibroma                 160
actinic keratosis              149
Name: count, dtype: int64

# Count diagnoses
diagnosis_counts = metadata['diagnosis'].value_counts().reset_index()
diagnosis_counts.columns = ['diagnosis', 'count']

# Plot
plt.figure(figsize=(10, 6))
sns.barplot(
    data=diagnosis_counts,
    x='count',
    y='diagnosis',
    hue='diagnosis',  # This applies the palette per category
    dodge=False,
    palette="viridis",
    legend=False
)
plt.title("Distribution of Diagnoses")
plt.xlabel("Count")
plt.ylabel("Diagnosis")
plt.tight_layout()
plt.show()

# Get all diagnosis counts
diagnosis_counts = metadata['diagnosis'].value_counts()

# Plot
plt.figure(figsize=(10, 10))
plt.pie(
    diagnosis_counts.values,
    labels=diagnosis_counts.index,
    autopct='%1.1f%%',
    startangle=140
)
plt.title("Diagnosis Distribution ")
plt.axis('equal')
plt.show()

# Identify the rows where 'diagnosis_1' is 'Indeterminate'
rows_to_drop = metadata[metadata['diagnosis_1'] == 'Indeterminate'].index

# Drop the rows from the dataframe
metadata.drop(rows_to_drop, inplace=True)

#reset the index
metadata.reset_index(drop=True, inplace=True)

# Define the label mapping function
def map_labels(row):
    # Nevus mapping
    if row["diagnosis"] == "nevus":
        return 0  # Nevus
    
    # Melanoma mapping
    elif row["diagnosis"] == "melanoma":
        return 1  # Melanoma
    
    # If diagnosis is from other specific types and diagnosis_1 is Benign, classify as Nevus
    elif (row["diagnosis"] in ["pigmented benign keratosis", "basal cell carcinoma", "squamous cell carcinoma", 
                               "vascular lesion", "dermatofibroma", "actinic keratosis"]) and row["diagnosis_1"] == "Benign":
        return 0  # Nevus
    
    # For Other lesion (Malignant)
    elif row["diagnosis"] == "Other lesion":
        if row["diagnosis_1"] in ["Malignant"]:
            return 2  # Other lesion (Malignant)
    
    # If diagnosis is one of the other types and diagnosis_1 is Malignant, classify as Other lesion
    if row["diagnosis_1"] in ["Malignant"]:
        return 2  # Other lesion

    return 2  # Default to Other lesion for any other case

# Apply the mapping function to create a new column 'mapped_label'
metadata["mapped_label"] = metadata.apply(map_labels, axis=1)

# Print class distribution
class_distribution = metadata["mapped_label"].value_counts().sort_index()
class_labels = {
    0: "Nevus",
    1: "Melanoma",
    2: "Other lesion"
}

# Output the class distribution
print("Class Distribution:")
for label, count in class_distribution.items():
    print(f"{class_labels[label]}: {count}")

Class Distribution:
Nevus: 9415
Melanoma: 1305
Other lesion: 851

class ISICDataset(Dataset):
    def __init__(self, metadata_path, root_dir, transform=None):
        """
        Args:
            metadata_path (str): Path to the CSV file containing metadata.
            root_dir (str): Directory with all the images.
            transform (callable, optional): Transformations to be applied to images.
        """
        self.metadata = pd.read_csv(metadata_path)
        self.metadata["image_name"] = self.metadata["isic_id"] + ".jpg"  # Fix filenames
        self.root_dir = root_dir
        self.transform = transform

        # Apply the label mapping logic and add the 'mapped_label' column
        self.metadata["mapped_label"] = self.metadata.apply(map_labels, axis=1)

    def __len__(self):
        return len(self.metadata)

    def __getitem__(self, idx):
        img_name = self.metadata.iloc[idx]["image_name"]
        label = self.metadata.iloc[idx]["mapped_label"]  # Use 'mapped_label' instead of the old 'label'

        img_path = os.path.join(self.root_dir, img_name)
        
        # Open image
        image = Image.open(img_path).convert("RGB")  # Ensure 3 channels
        
        # Apply transformations
        if self.transform:
            image = self.transform(image)
        
        return image, label

# Define transformations (data augmentation)
transform = transforms.Compose([
    transforms.RandomHorizontalFlip(p=0.5),  # Flip 50% of images horizontally
    transforms.RandomRotation(degrees=15),   # Rotate images randomly by ±15 degrees
    transforms.RandomResizedCrop(224, scale=(0.8, 1.0)),  # Random crop + resize
    transforms.RandomAffine(degrees=0, scale=(0.9, 1.1)),  # Randomly zoom image, scale range between 90% to 110%
    transforms.ColorJitter(brightness=0.2, contrast=0.2, saturation=0.2, hue=0.1),  # Adjust colors
    transforms.ToTensor(),  # Convert image to tensor
    transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) # Normalize with ImageNet mean/std
 ])

dataset = ISICDataset(metadata_path, root_dir, transform=transform)
dataloader = DataLoader(dataset, batch_size=32, shuffle=True)

# Test if the dataloader works
for images, labels in dataloader:
    print(f"Batch of images: {images.shape}, Labels: {labels.shape}")
    break  # Check one batch

Batch of images: torch.Size([32, 3, 224, 224]), Labels: torch.Size([32])

# Define the label mapping
label_mapping = {0: "nevus", 1: "melanoma", 2: "other lesion"}

# Map the numeric labels to the string labels for better visualization
mapped_label_counts = dataset.metadata["mapped_label"].map(label_mapping).value_counts()

# Create the bar chart
plt.figure(figsize=(6, 4))
bars = plt.bar(mapped_label_counts.index, mapped_label_counts.values, color=['blue', 'gray', 'red'])

# Add count labels on top of each bar
for bar in bars:
    height = bar.get_height()
    plt.text(bar.get_x() + bar.get_width() / 2, height, f'{height}', ha='center', va='bottom', fontsize=10)

# Add labels and title
plt.xlabel("Lesion Type")
plt.ylabel("Count")
plt.title("Distribution of Lesion Types")
plt.xticks(ticks=range(len(mapped_label_counts.index)), labels=mapped_label_counts.index, rotation=45)
plt.show()

# Pie chart
plt.figure(figsize=(6, 6))
plt.pie(mapped_label_counts.values, labels=mapped_label_counts.index, autopct='%1.1f%%', colors=['blue', 'gray', 'red'])
plt.title("Proportion of Lesion Types")
plt.show()

columns_to_remove = ['attribution', 'copyright_license', 'image_type']  
metadata = metadata.drop(columns=columns_to_remove)  

# Display the first few rows to confirm
print(metadata.head())

        isic_id  age_approx anatom_site_general anatom_site_special  \
0  ISIC_0024306        45.0                 NaN                 NaN   
1  ISIC_0024307        50.0     lower extremity                 NaN   
2  ISIC_0024308        55.0                 NaN                 NaN   
3  ISIC_0024309        40.0                 NaN                 NaN   
4  ISIC_0024310        60.0      anterior torso                 NaN   

  benign_malignant  concomitant_biopsy diagnosis diagnosis_1  \
0           benign               False     nevus      Benign   
1           benign               False     nevus      Benign   
2           benign               False     nevus      Benign   
3           benign               False     nevus      Benign   
4        malignant                True  melanoma   Malignant   

                                       diagnosis_2    diagnosis_3  \
0                Benign melanocytic proliferations          Nevus   
1                Benign melanocytic proliferations          Nevus   
2                Benign melanocytic proliferations          Nevus   
3                Benign melanocytic proliferations          Nevus   
4  Malignant melanocytic proliferations (Melanoma)  Melanoma, NOS   

             diagnosis_confirm_type   lesion_id  melanocytic     sex  \
0  serial imaging showing no change  IL_7252831         True    male   
1  serial imaging showing no change  IL_6125741         True    male   
2  serial imaging showing no change  IL_3692653         True  female   
3  serial imaging showing no change  IL_0959663         True    male   
4                    histopathology  IL_8194852         True    male   

         image_name  mapped_label  
0  ISIC_0024306.jpg             0  
1  ISIC_0024307.jpg             0  
2  ISIC_0024308.jpg             0  
3  ISIC_0024309.jpg             0  
4  ISIC_0024310.jpg             1

# Check if any missing values remain
print(metadata.isnull().sum())

isic_id                       0
age_approx                  380
anatom_site_general        2158
anatom_site_special       11040
benign_malignant           2529
concomitant_biopsy            0
diagnosis                     0
diagnosis_1                   0
diagnosis_2                   0
diagnosis_3                 180
diagnosis_confirm_type        0
lesion_id                     0
melanocytic                   0
sex                         340
image_name                    0
mapped_label                  0
dtype: int64

# Fill missing values in categorical columns with 'Unknown'
metadata['anatom_site_general'] = metadata['anatom_site_general'].fillna('Unknown')
metadata['anatom_site_special'] = metadata['anatom_site_special'].fillna('Unknown')
metadata['benign_malignant'] = metadata['benign_malignant'].fillna('Unknown')


# Fill missing values in numerical columns like 'age_approx' with the median
metadata['age_approx'] = metadata['age_approx'].fillna(metadata['age_approx'].median())

# Fill missing values in categorical columns with 'Unknown' or the mode
metadata['diagnosis_3'] = metadata['diagnosis_3'].fillna('Unknown')
metadata['sex'] = metadata['sex'].fillna(metadata['sex'].mode()[0])

print(metadata.isnull().sum())  # Check if any columns still have missing values

isic_id                   0
age_approx                0
anatom_site_general       0
anatom_site_special       0
benign_malignant          0
concomitant_biopsy        0
diagnosis                 0
diagnosis_1               0
diagnosis_2               0
diagnosis_3               0
diagnosis_confirm_type    0
lesion_id                 0
melanocytic               0
sex                       0
image_name                0
mapped_label              0
dtype: int64

print(metadata.head())  # Check the first few rows of the DataFrame

        isic_id  age_approx anatom_site_general anatom_site_special  \
0  ISIC_0024306        45.0             Unknown             Unknown   
1  ISIC_0024307        50.0     lower extremity             Unknown   
2  ISIC_0024308        55.0             Unknown             Unknown   
3  ISIC_0024309        40.0             Unknown             Unknown   
4  ISIC_0024310        60.0      anterior torso             Unknown   

  benign_malignant  concomitant_biopsy diagnosis diagnosis_1  \
0           benign               False     nevus      Benign   
1           benign               False     nevus      Benign   
2           benign               False     nevus      Benign   
3           benign               False     nevus      Benign   
4        malignant                True  melanoma   Malignant   

                                       diagnosis_2    diagnosis_3  \
0                Benign melanocytic proliferations          Nevus   
1                Benign melanocytic proliferations          Nevus   
2                Benign melanocytic proliferations          Nevus   
3                Benign melanocytic proliferations          Nevus   
4  Malignant melanocytic proliferations (Melanoma)  Melanoma, NOS   

             diagnosis_confirm_type   lesion_id  melanocytic     sex  \
0  serial imaging showing no change  IL_7252831         True    male   
1  serial imaging showing no change  IL_6125741         True    male   
2  serial imaging showing no change  IL_3692653         True  female   
3  serial imaging showing no change  IL_0959663         True    male   
4                    histopathology  IL_8194852         True    male   

         image_name  mapped_label  
0  ISIC_0024306.jpg             0  
1  ISIC_0024307.jpg             0  
2  ISIC_0024308.jpg             0  
3  ISIC_0024309.jpg             0  
4  ISIC_0024310.jpg             1

label_encoder = LabelEncoder()
metadata['sex'] = label_encoder.fit_transform(metadata['sex'])

class CustomImageDataset(Dataset):
    def __init__(self, metadata_df, root_dir, transform=None):
        """
        Args:
            metadata_df (DataFrame): DataFrame containing image metadata.
            root_dir (str): Directory with all the images.
            transform (callable, optional): Optional transform to be applied on a sample.
        """
        self.metadata = metadata_df  # The DataFrame containing metadata
        self.metadata["mapped_label"] = self.metadata.apply(map_labels, axis=1)  # Map diagnosis to numeric labels (axis=1 applies to rows)
        self.root_dir = root_dir
        self.transform = transform

    def __len__(self):
        # Return the total number of images in the dataset
        return len(self.metadata)

    def __getitem__(self, idx):
        # Get image name (ensure you append the file extension like '.jpg')
        img_name = self.metadata.iloc[idx, 0] + ".jpg"  # Assuming the first column has image names without extension
        img_path = os.path.join(self.root_dir, img_name)  # Get full image path

        # Check if the file exists (optional but good practice)
        if not os.path.exists(img_path):
            raise FileNotFoundError(f"Image {img_name} not found in {self.root_dir}")

        # Load the image
        image = Image.open(img_path).convert("RGB")  # Open and convert to RGB if not already

        # Get the corresponding label (mapped label from the DataFrame)
        label = self.metadata.iloc[idx]["mapped_label"]

        # Apply transformations if provided (e.g., resize, normalize, etc.)
        if self.transform:
            image = self.transform(image)

        return image, label  # Return the image and its label

print(torch.cuda.is_available())  
print(torch.cuda.get_device_name(0))  
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")

True
NVIDIA GeForce RTX 4080 SUPER

# feature_extract is a boolean that defines if we are finetuning or feature extracting. 
# If feature_extract = False, the model is finetuned and all model parameters are updated. 
# If feature_extract = True, only the last layer parameters are updated, the others remain fixed.
def set_parameter_requires_grad(model, feature_extracting):
    if feature_extracting:
        for param in model.parameters():
            param.requires_grad = False

class_weights = compute_class_weight(
    class_weight="balanced",  # Automatically balance class weights
    classes=np.array([0, 1, 2]),  # Specify class labels explicitly
    y=metadata["mapped_label"].values  # Convert the 'mapped_label' column to a numpy array
)
class_weights = torch.tensor(class_weights, dtype=torch.float32).to(device)
print("Class Weights:", class_weights)
criterion = torch.nn.CrossEntropyLoss(weight=class_weights)  # Loss function for multi-class classification

# Load ResNet-50 model with updated weights parameter
model = models.resnet50(weights=ResNet50_Weights.DEFAULT)

set_parameter_requires_grad(model, feature_extracting=False)

# Modify the final fully connected layer
num_ftrs = model.fc.in_features
model.fc = nn.Sequential(
    nn.Linear(num_ftrs, 512),  # Fully connected layer with 512 units
    nn.ReLU(),                 # Apply ReLU activation
    nn.Dropout(0.5),           # Apply 50% dropout
    nn.Linear(512, 3)          # Final output layer with 3 classes ('nevus', 'melanoma', 'other lesion')
)

# Initialize optimizer with L2 regularization (weight decay)
# optimizer = torch.optim.SGD(model.parameters(), lr=0.003, momentum=0.9, weight_decay=1e-4) # L2 regularization through weight_decay

# Adam can be used instead If needed:

optimizer = Adam(
    model.parameters(),
    lr=0.0001,
    betas=(0.9, 0.999),
    eps=1e-7,
    amsgrad=False
)

Class Weights: tensor([0.4097, 2.9556, 4.5323], device='cuda:0')

# Split the data into training and validation sets (80% training, 20% validation)
train_metadata, val_metadata = train_test_split(metadata, test_size=0.2, random_state=42)

# Check the split sizes
print(f"Training set size: {len(train_metadata)}")
print(f"Validation set size: {len(val_metadata)}")

Training set size: 9256
Validation set size: 2315

# Initialize the CustomImageDataset with the DataFrame
train_dataset = CustomImageDataset(metadata_df=train_metadata, root_dir=root_dir, transform=transform)
val_dataset = CustomImageDataset(metadata_df=val_metadata, root_dir=root_dir, transform=transform)

# Function to apply Grad-CAM
class GradCAM:
    def __init__(self, model, target_layer):
        self.model = model
        self.target_layer = target_layer
        self.gradients = None
        self.activation = None

        # Hook to get gradients
        self.target_layer.register_forward_hook(self.save_activation)
        self.target_layer.register_backward_hook(self.save_gradient)

    def save_activation(self, module, input, output):
        self.activation = output

    def save_gradient(self, module, grad_input, grad_output):
        self.gradients = grad_output[0]

    def generate_cam(self, class_idx):
        grad_val = self.gradients.cpu().data.numpy()
        act_val = self.activation.cpu().data.numpy()

        weights = np.mean(grad_val, axis=(2, 3), keepdims=True)
        cam = np.sum(weights * act_val, axis=1)

        # Normalize and resize CAM
        cam = np.maximum(cam, 0)  # ReLU
        cam = cam[0]  # Get first item
        cam = cv2.resize(cam, (224, 224))  # Resize to match image

        # Normalize CAM
        cam = cam - np.min(cam)
        cam = cam / np.max(cam)

        return cam

# Define a function for the training loop
def train_one_epoch(model, train_loader, criterion, optimizer, device, epoch, clip_grad_norm=None):
    model.train()
    running_loss = 0.0
    correct_predictions = 0
    total_predictions = 0
    start_time = time.time()  # Track epoch time
    
    for i, (inputs, labels) in enumerate(train_loader):
        inputs, labels = inputs.to(device), labels.to(device)

        optimizer.zero_grad()
        outputs = model(inputs)
        loss = criterion(outputs, labels)
        loss.backward()
        
        # Gradient clipping (if applicable)
        if clip_grad_norm:
            torch.nn.utils.clip_grad_norm_(model.parameters(), clip_grad_norm)

        optimizer.step()

        running_loss += loss.item()
        _, predicted = torch.max(outputs, 1)
        correct_predictions += (predicted == labels).sum().item()
        total_predictions += labels.size(0)

        # Log learning rate every 100 iterations 
        if (i + 1) % 100 == 0:
            print(f"Epoch {epoch+1}, Iteration {i + 1}/{len(train_loader)}")


    # Calculate average loss and accuracy
    train_loss = running_loss / len(train_loader)
    train_accuracy = 100 * correct_predictions / total_predictions
    
    # Print epoch time
    epoch_time = time.time() - start_time
    print(f"Epoch {epoch+1} completed in {epoch_time:.2f} seconds")

    return train_loss, train_accuracy

# Define a function for the validation loop
def validate_model(model, val_loader, criterion, device, num_examples=5):
    model.eval()
    val_loss = 0.0
    val_correct_predictions = 0
    val_total_predictions = 0

    # Store misclassified images, true labels, and predicted labels
    misclassified_images = []
    true_labels = []
    predicted_labels = []

    with torch.no_grad():
        for inputs, labels in val_loader:
            inputs, labels = inputs.to(device), labels.to(device)
            outputs = model(inputs)
            loss = criterion(outputs, labels)
            val_loss += loss.item()

            _, predicted = torch.max(outputs, 1)
            val_correct_predictions += (predicted == labels).sum().item()
            val_total_predictions += labels.size(0)

            # Find misclassified images
            misclassified_mask = predicted != labels
            misclassified_images.extend(inputs[misclassified_mask].cpu())
            true_labels.extend(labels[misclassified_mask].cpu())
            predicted_labels.extend(predicted[misclassified_mask].cpu())

    val_loss /= len(val_loader)
    val_accuracy = 100 * val_correct_predictions / val_total_predictions

    return val_loss, val_accuracy, misclassified_images, true_labels, predicted_labels

# Initialize lists to store loss and accuracy
train_losses = []
val_losses = []
train_accuracies = []
val_accuracies = []

# Set up the DataLoader
batch_size = 32
train_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True)
val_loader = DataLoader(val_dataset, batch_size=batch_size, shuffle=False)

# Set early stopping parameters
patience = 6  # Number of epochs to wait for improvement
best_val_loss = float('inf')
best_Accuracy = float('inf')
epochs_without_improvement = 0

num_epochs = 20  # Set the number of epochs
# Set up device
model.to(device)

# Store misclassified images at the end
final_misclassified_images = []
final_true_labels = []
final_predicted_labels = []

# Training and validation loop
for epoch in range(num_epochs):
    train_loss, train_accuracy = train_one_epoch(model, train_loader, criterion, optimizer, device, epoch)
    train_losses.append(train_loss)
    train_accuracies.append(train_accuracy)

    print(f"Epoch [{epoch+1}/{num_epochs}], Loss: {train_loss:.4f}, Accuracy: {train_accuracy:.2f}%")

    val_loss, val_accuracy, misclassified_images, true_labels, predicted_labels = validate_model(model, val_loader, criterion, device)
    val_losses.append(val_loss)
    val_accuracies.append(val_accuracy)

    # Save the last batch of misclassified images
    final_misclassified_images = misclassified_images
    final_true_labels = true_labels
    final_predicted_labels = predicted_labels

    print(f"Validation Loss: {val_loss:.4f}, Validation Accuracy: {val_accuracy:.2f}%")

    # Early stopping check
    if val_loss < best_val_loss:
        best_val_loss = val_loss
        best_accuracy = val_accuracy
        epochs_without_improvement = 0
        torch.save(model.state_dict(), 'resnet50_custom_best.pth')
        torch.save(optimizer.state_dict(), 'resnet50_optimizer_best.pth')
        print("Best Model saved")
    else:
        epochs_without_improvement += 1
        if epochs_without_improvement >= patience:
            print("Early stopping triggered!")
            break

# Print final results
print(f"Finished Training and Validation, Best Model Accuracy: {best_accuracy:.2f}%, Loss: {best_val_loss:.4f}")

Epoch 1, Iteration 100/290
Epoch 1, Iteration 200/290
Epoch 1 completed in 72.73 seconds
Epoch [1/20], Loss: 0.6940, Accuracy: 69.58%
Validation Loss: 0.6664, Validation Accuracy: 72.40%
Best Model saved
Epoch 2, Iteration 100/290
Epoch 2, Iteration 200/290
Epoch 2 completed in 71.93 seconds
Epoch [2/20], Loss: 0.5269, Accuracy: 74.68%
Validation Loss: 0.4821, Validation Accuracy: 76.33%
Best Model saved
Epoch 3, Iteration 100/290
Epoch 3, Iteration 200/290
Epoch 3 completed in 71.65 seconds
Epoch [3/20], Loss: 0.4498, Accuracy: 77.28%
Validation Loss: 0.5578, Validation Accuracy: 84.41%
Epoch 4, Iteration 100/290
Epoch 4, Iteration 200/290
Epoch 4 completed in 72.07 seconds
Epoch [4/20], Loss: 0.4057, Accuracy: 79.62%
Validation Loss: 0.4737, Validation Accuracy: 73.78%
Best Model saved
Epoch 5, Iteration 100/290
Epoch 5, Iteration 200/290
Epoch 5 completed in 71.63 seconds
Epoch [5/20], Loss: 0.3706, Accuracy: 81.30%
Validation Loss: 0.5036, Validation Accuracy: 85.23%
Epoch 6, Iteration 100/290
Epoch 6, Iteration 200/290
Epoch 6 completed in 71.93 seconds
Epoch [6/20], Loss: 0.3454, Accuracy: 82.49%
Validation Loss: 0.5386, Validation Accuracy: 83.28%
Epoch 7, Iteration 100/290
Epoch 7, Iteration 200/290
Epoch 7 completed in 71.80 seconds
Epoch [7/20], Loss: 0.2949, Accuracy: 84.78%
Validation Loss: 0.4169, Validation Accuracy: 80.69%
Best Model saved
Epoch 8, Iteration 100/290
Epoch 8, Iteration 200/290
Epoch 8 completed in 72.00 seconds
Epoch [8/20], Loss: 0.2725, Accuracy: 86.06%
Validation Loss: 0.4407, Validation Accuracy: 79.65%
Epoch 9, Iteration 100/290
Epoch 9, Iteration 200/290
Epoch 9 completed in 71.53 seconds
Epoch [9/20], Loss: 0.2494, Accuracy: 86.73%
Validation Loss: 0.4718, Validation Accuracy: 84.79%
Epoch 10, Iteration 100/290
Epoch 10, Iteration 200/290
Epoch 10 completed in 71.83 seconds
Epoch [10/20], Loss: 0.2502, Accuracy: 87.06%
Validation Loss: 0.5192, Validation Accuracy: 86.09%
Epoch 11, Iteration 100/290
Epoch 11, Iteration 200/290
Epoch 11 completed in 71.84 seconds
Epoch [11/20], Loss: 0.2191, Accuracy: 88.81%
Validation Loss: 0.4157, Validation Accuracy: 80.52%
Best Model saved
Epoch 12, Iteration 100/290
Epoch 12, Iteration 200/290
Epoch 12 completed in 71.73 seconds
Epoch [12/20], Loss: 0.2021, Accuracy: 88.95%
Validation Loss: 0.5886, Validation Accuracy: 87.04%
Epoch 13, Iteration 100/290
Epoch 13, Iteration 200/290
Epoch 13 completed in 71.73 seconds
Epoch [13/20], Loss: 0.2071, Accuracy: 89.82%
Validation Loss: 0.4686, Validation Accuracy: 85.14%
Epoch 14, Iteration 100/290
Epoch 14, Iteration 200/290
Epoch 14 completed in 71.93 seconds
Epoch [14/20], Loss: 0.1654, Accuracy: 91.40%
Validation Loss: 0.6447, Validation Accuracy: 85.70%
Epoch 15, Iteration 100/290
Epoch 15, Iteration 200/290
Epoch 15 completed in 71.50 seconds
Epoch [15/20], Loss: 0.1775, Accuracy: 90.91%
Validation Loss: 0.4318, Validation Accuracy: 85.83%
Epoch 16, Iteration 100/290
Epoch 16, Iteration 200/290
Epoch 16 completed in 71.68 seconds
Epoch [16/20], Loss: 0.1525, Accuracy: 92.34%
Validation Loss: 0.4751, Validation Accuracy: 84.79%
Epoch 17, Iteration 100/290
Epoch 17, Iteration 200/290
Epoch 17 completed in 71.65 seconds
Epoch [17/20], Loss: 0.1455, Accuracy: 92.25%
Validation Loss: 0.5289, Validation Accuracy: 88.94%
Early stopping triggered!
Finished Training and Validation, Best Model Accuracy: 80.52%, Loss: 0.4157

# Plot Training & Validation Loss
plt.figure(figsize=(10, 5))
plt.plot(range(1, len(train_losses) + 1), train_losses, label="Training Loss", marker='o')
plt.plot(range(1, len(val_losses) + 1), val_losses, label="Validation Loss", marker='o')
plt.xlabel("Epochs")
plt.ylabel("Loss")
plt.title("Training & Validation Loss")
plt.legend()
plt.grid()
plt.show()

# Plot Training & Validation Accuracy
plt.figure(figsize=(10, 5))
plt.plot(range(1, len(train_accuracies) + 1), train_accuracies, label="Training Accuracy", marker='o')
plt.plot(range(1, len(val_accuracies) + 1), val_accuracies, label="Validation Accuracy", marker='o')
plt.xlabel("Epochs")
plt.ylabel("Accuracy (%)")
plt.title("Training & Validation Accuracy")
plt.legend()
plt.grid()
plt.show()

# Define a mapping for label names
label_mapping = {0: "nevus", 1: "melanoma", 2: "other lesion"}

model.eval()
y_true = []
y_pred = []

with torch.no_grad():
    for images, labels in val_loader:
        images = images.to(device)
        labels = labels.to(device)
        outputs = model(images)
        predictions = outputs.max(1)[1]  # Get predicted class index

        y_true.extend(labels.cpu().numpy())
        y_pred.extend(predictions.cpu().numpy())

# Compute confusion matrix 
confusion_mtx = confusion_matrix(y_true, y_pred)

# Function to plot confusion matrix with COUNT values
def plot_confusion_matrix(cm, classes, normalize=False, title="Confusion Matrix", cmap=plt.cm.Blues):
    plt.figure(figsize=(8, 6))
    plt.imshow(cm, interpolation='nearest', cmap=cmap)
    plt.title(title)
    plt.colorbar()
    
    tick_marks = np.arange(len(classes))
    plt.xticks(tick_marks, classes, rotation=45)
    plt.yticks(tick_marks, classes)

    if normalize:
        cm = cm.astype("float") / cm.sum(axis=1)[:, np.newaxis]  # Normalize by row sum

    # Define threshold for color contrast
    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):
        color = "green" if i == j else "red"  # Green for correct, Red for incorrect
        plt.text(j, i, f"{cm[i, j]:.2f}" if normalize else f"{cm[i, j]}",
                 horizontalalignment="center",
                 color=color, fontsize=12, fontweight="bold")

    plt.ylabel("True label")
    plt.xlabel("Predicted label")
    plt.tight_layout()
    plt.show()

# Plot confusion matrix as COUNT instead of percentage
plot_confusion_matrix(confusion_mtx, list(label_mapping.values()), normalize=False, title="Confusion Matrix")


if len(final_misclassified_images) > 0:
    print(f"Showing {min(5, len(final_misclassified_images))} misclassified images from final validation:")
    for i in range(min(5, len(final_misclassified_images))):
        image = final_misclassified_images[i]
        true_label = label_mapping[final_true_labels[i].item()]
        predicted_label = label_mapping[final_predicted_labels[i].item()]

        # Convert tensor to numpy and denormalize
        image = image.permute(1, 2, 0).numpy() 
        if image.min() < 0 or image.max() > 1: 
            image = (image - image.min()) / (image.max() - image.min())

        # Plot the image
        plt.figure(figsize=(5, 5))
        plt.imshow(image)
        plt.title(f"True: {true_label}, Predicted: {predicted_label}")
        plt.axis("off")
        plt.show()

Showing 5 misclassified images from final validation:

# Path to the directory containing images
image_dir = os.path.join(MEDIA_ROOT, 'images_ToTest', 'images')

# Get all image files in the directory
image_files = [f for f in os.listdir(image_dir) if f.endswith(('.png', '.jpg', '.jpeg'))]

# Set model to evaluation mode
model.eval()

# Define confidence threshold (adjust as needed)
confidence_threshold = 70  # Threshold in percentage 

# Class labels
class_labels = ['nevus', 'melanoma', 'other lesion']

# Choose the layer for ResNet (last conv layer in layer4)
target_layer = model.layer4[-1] 

# Initialize Grad-CAM
grad_cam = GradCAM(model=model, target_layer=target_layer)

# Iterate through all images
for image_file in image_files:
    image_path = os.path.join(image_dir, image_file)
    
    # Load and preprocess the image
    image = Image.open(image_path)
    input_tensor = transform(image).unsqueeze(0).to(device)  # Add batch dimension and move to device
    input_tensor.requires_grad_(True)  # Enable gradient computation

    # Forward pass
    output = model(input_tensor)
    probabilities = torch.nn.functional.softmax(output, dim=1)  # Get softmax probabilities
    confidence, predicted = torch.max(probabilities, 1)  # Get max probability and index

    # Perform backward pass to compute gradients
    model.zero_grad()  # Clear existing gradients
    output[0, predicted.item()].backward()  # Backward pass for the predicted class

    predicted_class = class_labels[predicted.item()]
    confidence_percentage = confidence.item() * 100  # Convert to percentage

    # Extract image ID from filename 
    image_id = os.path.splitext(image_file)[0]  

    # Generate the Grad-CAM heatmap
    cam = grad_cam.generate_cam(predicted.item())  # Pass the predicted class index

    # Convert original image to NumPy array
    image_np = np.array(image.resize((224, 224)))

    # Squeeze the CAM to ensure it's 2D
    cam = np.squeeze(cam)

    # Normalize and convert cam to uint8
    cam = cam - np.min(cam)  # Shift values to positive range
    cam = cam / np.max(cam)  # Normalize to [0,1]
    cam = np.uint8(255 * cam)  # Scale to [0,255] and convert to uint8

    # Overlay heatmap on image
    heatmap = cv2.applyColorMap(cam, cv2.COLORMAP_JET)
    superimposed_img = cv2.addWeighted(image_np, 0.6, heatmap, 0.4, 0)

    # Print results with percentage confidence and display Grad-CAM
    if confidence_percentage < confidence_threshold:
        print(f"Image ID: {image_id}, Prediction Uncertain (Confidence: {confidence_percentage:.2f}%)")
    else:
        print(f"Image ID: {image_id}, Predicted Outcome: {predicted_class} (Confidence: {confidence_percentage:.2f}%)")

    # Display the original and Grad-CAM result
    plt.figure(figsize=(10, 5))
    plt.subplot(1, 2, 1)
    plt.imshow(image_np)
    plt.title(f"Original Image: {image_id}")

    plt.subplot(1, 2, 2)
    plt.imshow(superimposed_img)
    plt.title("Grad-CAM Heatmap")
    

    plt.show()

C:\Users\mantr\AppData\Local\Programs\Python\Python312\Lib\site-packages\torch\nn\modules\module.py:1830: FutureWarning: Using a non-full backward hook when the forward contains multiple autograd Nodes is deprecated and will be removed in future versions. This hook will be missing some grad_input. Please use register_full_backward_hook to get the documented behavior.
  self._maybe_warn_non_full_backward_hook(args, result, grad_fn)

Image ID: Car_Image, Predicted Outcome: nevus (Confidence: 89.98%)

Image ID: ISIC_1871644, Predicted Outcome: nevus (Confidence: 99.96%)

Image ID: ISIC_2175344, Predicted Outcome: nevus (Confidence: 99.05%)

Image ID: ISIC_2310644, Predicted Outcome: melanoma (Confidence: 71.87%)

Image ID: ISIC_5178159, Predicted Outcome: melanoma (Confidence: 99.46%)

Image ID: ISIC_5562564, Predicted Outcome: nevus (Confidence: 97.37%)

Image ID: ISIC_6377613, Predicted Outcome: other lesion (Confidence: 99.39%)

Image ID: ISIC_6821316, Predicted Outcome: melanoma (Confidence: 99.28%)

Image ID: ISIC_7522225, Predicted Outcome: nevus (Confidence: 99.96%)

This is a Project about skin cancer detection using AI¶

First we import all the libraries we will be using¶

We will be loading our dataset which contain about 10,000 images from ISIC(HAM10000)¶

now we will print all the labels and the count for the label "diagnosis"¶

Visualize how common each skin condition is in the dataset using a horizontal bar chart and pie chart¶

Mapping Lesions to Numeric Labels¶

Creating and Loading the ISICDataset for Skin Lesion Classification¶

Visualizing the Distribution of Lesion Types¶

Data Preprocessing: Removing Unnecessary Columns and Handling Missing Values¶

CustomImageDataset: A PyTorch Dataset for Loading and Transforming Images¶

run the model on the device CPU/GPU¶

Setting Up ResNet-50 with Class Weights and Optimization¶

Training the Model with Early Stopping and Validation¶

Visualizing Model Performance: Loss & Accuracy Trends¶

Confusion Matrix and Misclassified Images Analysis¶

Grad-CAM Visualization for Model Predictions¶